{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.使用 lightGBM 预测音乐推荐结果     测试数据特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "import re\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../Data/'  # 文件路径\n",
    "model_path = '../model/' # 模型路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有字段都转换成了类别特征的特征数据 加载到内存\n",
    "# 不包含这些特征 ['artist_name','composer','lyricist','name','genre_ids']\n",
    "with open(model_path + 'data_all_test_v2.pkl','rb') as fr:\n",
    "    data_all_test_v2 = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76126</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790668</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52111</th>\n",
       "      <td>discover</td>\n",
       "      <td>screen_name_NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159296</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160750</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       source_system_tab   source_screen_name          source_type city bd  \\\n",
       "76126         my library  Local playlist more        local-library    1  1   \n",
       "790668        my library  Local playlist more        local-library    1  1   \n",
       "52111           discover      screen_name_NaN  song-based-playlist    1  1   \n",
       "159296             radio                Radio                radio    3  3   \n",
       "160750             radio                Radio                radio    3  3   \n",
       "\n",
       "            gender registered_via song_length language reg_interval_days  \n",
       "76126   gender_NaN              7           1        3                 3  \n",
       "790668  gender_NaN              7           2        3                 3  \n",
       "52111   gender_NaN              4           2       17                 0  \n",
       "159296        male              9           1       52                10  \n",
       "160750        male              9           1       -1                10  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2556790 entries, 76126 to 86415\n",
      "Data columns (total 10 columns):\n",
      "source_system_tab     category\n",
      "source_screen_name    category\n",
      "source_type           category\n",
      "city                  category\n",
      "bd                    category\n",
      "gender                category\n",
      "registered_via        category\n",
      "song_length           category\n",
      "language              category\n",
      "reg_interval_days     category\n",
      "dtypes: category(10)\n",
      "memory usage: 43.9 MB\n"
     ]
    }
   ],
   "source": [
    "data_all_test_v2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 10)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有字段都转换成了类别特征的特征数据 加载到内存\n",
    "with open(model_path + 'data_all_test.pkl','rb') as fr:\n",
    "    data_all_test = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>registration_init_time</th>\n",
       "      <th>expiration_date</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>composer</th>\n",
       "      <th>lyricist</th>\n",
       "      <th>language</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76126</th>\n",
       "      <td>0</td>\n",
       "      <td>V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=</td>\n",
       "      <td>WmHKgKMlp1lQMecNdNvDMkvIycZYHnFwDT72I5sIssc=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>20160219</td>\n",
       "      <td>20170918</td>\n",
       "      <td>224130.0</td>\n",
       "      <td>458</td>\n",
       "      <td>梁文音 (Rachel Liang)</td>\n",
       "      <td>Qi Zheng Zhang</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>愛其實很殘忍</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790668</th>\n",
       "      <td>1</td>\n",
       "      <td>V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=</td>\n",
       "      <td>y/rsZ9DC7FwK5F2PK2D5mj+aOBUJAjuu3dZ14NgE0vM=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>20160219</td>\n",
       "      <td>20170918</td>\n",
       "      <td>320470.0</td>\n",
       "      <td>465</td>\n",
       "      <td>林俊傑 (JJ Lin)</td>\n",
       "      <td>林俊傑</td>\n",
       "      <td>孫燕姿/易家揚</td>\n",
       "      <td>3.0</td>\n",
       "      <td>她說</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52111</th>\n",
       "      <td>2</td>\n",
       "      <td>/uQAlrAkaczV+nWCd2sPF2ekvXPRipV7q0l+gbLuxjw=</td>\n",
       "      <td>8eZLFOdGVdXBSqoAv5nsLigeH2BvKXzTQYtUM53I0k4=</td>\n",
       "      <td>discover</td>\n",
       "      <td>NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>20161117</td>\n",
       "      <td>20161124</td>\n",
       "      <td>315899.0</td>\n",
       "      <td>2022</td>\n",
       "      <td>Yu Takahashi (高橋優)</td>\n",
       "      <td>Yu Takahashi</td>\n",
       "      <td>Yu Takahashi</td>\n",
       "      <td>17.0</td>\n",
       "      <td>subarashiki nichijo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159296</th>\n",
       "      <td>3</td>\n",
       "      <td>1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=</td>\n",
       "      <td>ztCf8thYsS4YN3GcIL/bvoxLm/T5mYBVKOO4C9NiVfQ=</td>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>20070725</td>\n",
       "      <td>20170430</td>\n",
       "      <td>285210.0</td>\n",
       "      <td>465</td>\n",
       "      <td>U2</td>\n",
       "      <td>The Edge| Adam Clayton| Larry Mullen| Jr.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>Hold Me| Thrill Me| Kiss Me| Kill Me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160750</th>\n",
       "      <td>4</td>\n",
       "      <td>1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=</td>\n",
       "      <td>MKVMpslKcQhMaFEgcEQhEfi5+RZhMYlU3eRDpySrH8Y=</td>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>20070725</td>\n",
       "      <td>20170430</td>\n",
       "      <td>197590.0</td>\n",
       "      <td>873</td>\n",
       "      <td>Yoga Mr Sound</td>\n",
       "      <td>Neuromancer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Om Yoga</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                          msno  \\\n",
       "76126    0  V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=   \n",
       "790668   1  V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=   \n",
       "52111    2  /uQAlrAkaczV+nWCd2sPF2ekvXPRipV7q0l+gbLuxjw=   \n",
       "159296   3  1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=   \n",
       "160750   4  1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=   \n",
       "\n",
       "                                             song_id source_system_tab  \\\n",
       "76126   WmHKgKMlp1lQMecNdNvDMkvIycZYHnFwDT72I5sIssc=        my library   \n",
       "790668  y/rsZ9DC7FwK5F2PK2D5mj+aOBUJAjuu3dZ14NgE0vM=        my library   \n",
       "52111   8eZLFOdGVdXBSqoAv5nsLigeH2BvKXzTQYtUM53I0k4=          discover   \n",
       "159296  ztCf8thYsS4YN3GcIL/bvoxLm/T5mYBVKOO4C9NiVfQ=             radio   \n",
       "160750  MKVMpslKcQhMaFEgcEQhEfi5+RZhMYlU3eRDpySrH8Y=             radio   \n",
       "\n",
       "         source_screen_name          source_type  city  bd gender  \\\n",
       "76126   Local playlist more        local-library     1   0    NaN   \n",
       "790668  Local playlist more        local-library     1   0    NaN   \n",
       "52111                   NaN  song-based-playlist     1   0    NaN   \n",
       "159296                Radio                radio     3  30   male   \n",
       "160750                Radio                radio     3  30   male   \n",
       "\n",
       "        registered_via  registration_init_time  expiration_date  song_length  \\\n",
       "76126                7                20160219         20170918     224130.0   \n",
       "790668               7                20160219         20170918     320470.0   \n",
       "52111                4                20161117         20161124     315899.0   \n",
       "159296               9                20070725         20170430     285210.0   \n",
       "160750               9                20070725         20170430     197590.0   \n",
       "\n",
       "       genre_ids         artist_name  \\\n",
       "76126        458  梁文音 (Rachel Liang)   \n",
       "790668       465        林俊傑 (JJ Lin)   \n",
       "52111       2022  Yu Takahashi (高橋優)   \n",
       "159296       465                  U2   \n",
       "160750       873       Yoga Mr Sound   \n",
       "\n",
       "                                         composer      lyricist  language  \\\n",
       "76126                              Qi Zheng Zhang           NaN       3.0   \n",
       "790668                                        林俊傑       孫燕姿/易家揚       3.0   \n",
       "52111                                Yu Takahashi  Yu Takahashi      17.0   \n",
       "159296  The Edge| Adam Clayton| Larry Mullen| Jr.           NaN      52.0   \n",
       "160750                                Neuromancer           NaN      -1.0   \n",
       "\n",
       "                                        name  \n",
       "76126                                 愛其實很殘忍  \n",
       "790668                                    她說  \n",
       "52111                    subarashiki nichijo  \n",
       "159296  Hold Me| Thrill Me| Kiss Me| Kill Me  \n",
       "160750                               Om Yoga  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 19)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'artist_name',\n",
       " 'composer',\n",
       " 'expiration_date',\n",
       " 'genre_ids',\n",
       " 'id',\n",
       " 'lyricist',\n",
       " 'msno',\n",
       " 'name',\n",
       " 'registration_init_time',\n",
       " 'song_id'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data_all_test.columns)-set(data_all_test_v2.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in ['artist_name','composer','lyricist','name']:\n",
    "    data_all_test[column] = data_all_test[column].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in ['artist_name','composer','genre_ids','lyricist','name']:\n",
    "    data_all_test_v2[column] = data_all_test[column]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>composer</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>lyricist</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76126</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>梁文音 (Rachel Liang)</td>\n",
       "      <td>Qi Zheng Zhang</td>\n",
       "      <td>458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>愛其實很殘忍</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790668</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>林俊傑 (JJ Lin)</td>\n",
       "      <td>林俊傑</td>\n",
       "      <td>465</td>\n",
       "      <td>孫燕姿/易家揚</td>\n",
       "      <td>她說</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52111</th>\n",
       "      <td>discover</td>\n",
       "      <td>screen_name_NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>Yu Takahashi (高橋優)</td>\n",
       "      <td>Yu Takahashi</td>\n",
       "      <td>2022</td>\n",
       "      <td>Yu Takahashi</td>\n",
       "      <td>subarashiki nichijo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159296</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>10</td>\n",
       "      <td>U2</td>\n",
       "      <td>The Edge| Adam Clayton| Larry Mullen| Jr.</td>\n",
       "      <td>465</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hold Me| Thrill Me| Kiss Me| Kill Me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160750</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>10</td>\n",
       "      <td>Yoga Mr Sound</td>\n",
       "      <td>Neuromancer</td>\n",
       "      <td>873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Om Yoga</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       source_system_tab   source_screen_name          source_type city bd  \\\n",
       "76126         my library  Local playlist more        local-library    1  1   \n",
       "790668        my library  Local playlist more        local-library    1  1   \n",
       "52111           discover      screen_name_NaN  song-based-playlist    1  1   \n",
       "159296             radio                Radio                radio    3  3   \n",
       "160750             radio                Radio                radio    3  3   \n",
       "\n",
       "            gender registered_via song_length language reg_interval_days  \\\n",
       "76126   gender_NaN              7           1        3                 3   \n",
       "790668  gender_NaN              7           2        3                 3   \n",
       "52111   gender_NaN              4           2       17                 0   \n",
       "159296        male              9           1       52                10   \n",
       "160750        male              9           1       -1                10   \n",
       "\n",
       "               artist_name                                   composer  \\\n",
       "76126   梁文音 (Rachel Liang)                             Qi Zheng Zhang   \n",
       "790668        林俊傑 (JJ Lin)                                        林俊傑   \n",
       "52111   Yu Takahashi (高橋優)                               Yu Takahashi   \n",
       "159296                  U2  The Edge| Adam Clayton| Larry Mullen| Jr.   \n",
       "160750       Yoga Mr Sound                                Neuromancer   \n",
       "\n",
       "       genre_ids      lyricist                                  name  \n",
       "76126        458           NaN                                愛其實很殘忍  \n",
       "790668       465       孫燕姿/易家揚                                    她說  \n",
       "52111       2022  Yu Takahashi                   subarashiki nichijo  \n",
       "159296       465           NaN  Hold Me| Thrill Me| Kiss Me| Kill Me  \n",
       "160750       873           NaN                               Om Yoga  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 15)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2556790 entries, 76126 to 86415\n",
      "Data columns (total 15 columns):\n",
      "source_system_tab     category\n",
      "source_screen_name    category\n",
      "source_type           category\n",
      "city                  category\n",
      "bd                    category\n",
      "gender                category\n",
      "registered_via        category\n",
      "song_length           category\n",
      "language              category\n",
      "reg_interval_days     category\n",
      "artist_name           category\n",
      "composer              category\n",
      "genre_ids             object\n",
      "lyricist              category\n",
      "name                  category\n",
      "dtypes: category(14), object(1)\n",
      "memory usage: 104.0+ MB\n"
     ]
    }
   ],
   "source": [
    "data_all_test_v2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将连续型变量离散化转为类别变量\n",
    "# 将原有的 str 类型的变量转换为类别变量\n",
    "with open(model_path+'data_all_test_v3.pkl', 'wb') as fw:\n",
    "    pk.dump(data_all_test_v2,fw)\n",
    "fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path+'genre_id_test_df.pkl', 'rb') as fr:\n",
    "    genre_id_df = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in genre_id_df.columns:\n",
    "    data_all_test_v2[column]=genre_id_df[column]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_153</th>\n",
       "      <th>genre_id_154</th>\n",
       "      <th>genre_id_155</th>\n",
       "      <th>genre_id_156</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76126</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790668</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52111</th>\n",
       "      <td>discover</td>\n",
       "      <td>screen_name_NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159296</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160750</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 178 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       source_system_tab   source_screen_name          source_type city bd  \\\n",
       "76126         my library  Local playlist more        local-library    1  1   \n",
       "790668        my library  Local playlist more        local-library    1  1   \n",
       "52111           discover      screen_name_NaN  song-based-playlist    1  1   \n",
       "159296             radio                Radio                radio    3  3   \n",
       "160750             radio                Radio                radio    3  3   \n",
       "\n",
       "            gender registered_via song_length language reg_interval_days  ...  \\\n",
       "76126   gender_NaN              7           1        3                 3  ...   \n",
       "790668  gender_NaN              7           2        3                 3  ...   \n",
       "52111   gender_NaN              4           2       17                 0  ...   \n",
       "159296        male              9           1       52                10  ...   \n",
       "160750        male              9           1       -1                10  ...   \n",
       "\n",
       "       genre_id_153 genre_id_154 genre_id_155 genre_id_156 genre_id_157  \\\n",
       "76126             0            0            0            0            0   \n",
       "790668            0            0            0            0            0   \n",
       "52111             0            0            0            0            0   \n",
       "159296            0            0            0            0            0   \n",
       "160750            0            0            0            0            0   \n",
       "\n",
       "        genre_id_158  genre_id_159  genre_id_160  genre_id_161  genre_id_162  \n",
       "76126              0             0             0             0             0  \n",
       "790668             0             0             0             0             0  \n",
       "52111              0             0             0             0             0  \n",
       "159296             0             0             0             0             0  \n",
       "160750             0             0             0             0             0  \n",
       "\n",
       "[5 rows x 178 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all_test_v2 = data_all_test_v2.drop(['genre_ids'],axis=1) # 对于 lightGBM 模型来说 genre_ids 字段如何处理比较合适"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 177)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path+'data_all_train_v4.pkl', 'rb') as fr:\n",
    "    data_all_train_v2 = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'genre_id_163', 'genre_id_164', 'genre_id_165', 'genre_id_166'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_test_genre_id_diff = set(data_all_train_v2.columns)-set(data_all_test_v2.columns)\n",
    "train_test_genre_id_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data_all_test_v2.columns)-set(data_all_train_v2.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 不同的字段以训练集为准"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in train_test_genre_id_diff:\n",
    "    data_all_test_v2[column] = np.zeros(data_all_test_v2.shape[0],dtype='uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set(data_all_train_v2.columns)-set(data_all_test_v2.columns)\n",
    "set(data_all_test_v2.columns)-set(data_all_train_v2.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 181)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_test_v2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将拼接了 genre_ids 离散化后的数据保存到磁盘，其他特征都已经转为类别特征\n",
    "# 拼接了 genre_ids 离散化后的数据保存到磁盘，其他特征都已经转为类别特征    （快）\n",
    "with open(model_path + 'data_all_test_v4.pkl','wb') as fw:\n",
    "    pk.dump(data_all_test_v2,fw)\n",
    "fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-27 22:58:23\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "nowTime=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')#现在\n",
    "print(nowTime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
